# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import codecs
import json
import pymongo
from scrapy.conf import settings
from scrapy.exceptions import DropItem
from datetime import datetime


# 去重复的 company
class DuplicatesPipeline(object):

    def __init__(self):
        self.ids_seen = set()

    def process_item(self, item, spider):
        if item['info_id'] in self.ids_seen:
            raise DropItem("Duplicate item found: %s" % item)
        else:
            self.ids_seen.add(item['info_id'])
            return item


class SpiderPipeline(object):
    def __init__(self):
        self.filename = codecs.open("test.json", "w", encoding="utf-8")
        pass

    def process_item(self, item, spider):
        # 格林威治时间，距离中国 +8 时区
        item['time'] = datetime.utcnow()
        # 爬虫名
        item['spidername'] = spider.name
        # 中文默认使用ascii码来存储，禁用后默认为Unicode字符串
        content = json.dumps(dict(item), ensure_ascii=False) + "\n"
        self.filename.write(content)
        return item

    def close_spider(self, spider):
        self.filename.close()
        pass


class MongodbPipeline(object):
    def __init__(self):
        host = settings["MONGODB_HOST"]
        port = settings["MONGODB_PORT"]
        dbname = settings["MONGODB_DBNAME"]
        sheetname = settings["MONGODB_SHEETNAME"]

        # 创建MONGODB数据库链接
        client = pymongo.MongoClient(host=host, port=port)
        # 指定数据库
        mydb = client[dbname]
        # 存放数据的数据库表名
        self.sheet = mydb[sheetname]

    def process_item(self, item, spider):
        data = dict(item)
        self.sheet.insert(data)
        return item


class MysqlPipeline(object):
    def __init__(self):
        pass

    def process_item(self, item, spider):
        return item
